// KeywordProcessor.h
#pragma once
#include <cppjieba/Jieba.hpp>
#include <utfcpp/utf8.h>
#include <string>
#include <set>
#include <iostream>
#include <fstream>
#include <string>
#include "DirectoryScanner.h"
#include <map>
#include "TextUtils.h"
#include <sstream>
#include <algorithm>
#include <thread>
#include <chrono>

using std::sort;
using std::set;
using std::map;
using std::string;
using std::ifstream;
using std::ofstream;
using std::cout;
using std::cerr;
using std::endl;
using std::stringstream;
using std::istringstream;
using std::ostringstream;

// #define DICT_CN "corpus/dict_cn.dat"
// #define INDEX_CN "corpus/index_cn.dat"
// #define DICT_EN "corpus/dict_en.dat"
// #define INDEX_EN "corpus/index_en.dat"

class KeywordProcessor {
public:
    KeywordProcessor();
	
    // chDir: 中文语料库
    // enDir: 英文语料库
    void process(const string& chDir, const string& enDir);

private:
    void create_cn_dict(const string& dir, const string& outfile);
    void build_cn_index(const string& dict, const string& index);

    void create_en_dict(const string& dir, const string& outfile);
    void build_en_index(const string& dict, const string& index);
    
private:
    cppjieba::Jieba m_tokenizer;
    set<string> m_enStopWords;
    set<string> m_chStopWords;
};
